package com.eric.r2d2.urlbase;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.util.HashSet;
import java.util.Map;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

public class UrlBasePush {
	static String save_prefix = "url_";
	static int nodes = 5000;

	public static void main(String[] args) throws IOException {
		long a = System.currentTimeMillis();
		if (args.length < 3) {
			System.out.println("Usage:UrlBasePush Url_Base_File_In Dir_out Dir_Seed_files");
			System.exit(1);
		}

		//输入文件
		File file_in = new File(args[0]);
		//定义输出目录
		File dir_out = new File(args[1]);
		if (!dir_out.exists()) dir_out.mkdirs();
		//定义seed files
		File dir_seeds = new File(args[2]);
		File[] seed_files = dir_seeds.listFiles();
		Map<Integer,HashSet<String>> seed_map = Prioritizer.readSeed(seed_files);

		BufferedReader br_in = new BufferedReader(new FileReader(file_in));
		BufferedWriter bw = null;
		
		//读取条目，对url进行hash，得出save_id，然后保存到对应的save_id文件
		String line_in;
		while ((line_in = br_in.readLine()) != null) {
			JSONObject json = JSON.parseObject(line_in);
			
			//计算save_id
			int save_id = (JSHash(json.getString("url"))) % nodes;
			File target_file = new File(dir_out + File.separator + save_prefix + save_id);
			if (!target_file.exists()) 	target_file.createNewFile();

			// 判断是否在目标文件中
			//速度太慢，在shell中用sort -u进行
			//if (isInTargetFile(line_in, dir_out + File.separator + save_prefix + save_id) == false) {
				bw = new BufferedWriter(new FileWriter(target_file, true));
				bw.append(Prioritizer.PrioritizeByDomain(json,seed_map).toJSONString());
				bw.append("\n");
				bw.flush();
				bw.close();
			//}
		}
		br_in.close();
		System.out.println(System.currentTimeMillis() - a);
	}

	public static int JSHash(String str) {
		int hash = 1315423911;
		for (int i = 0; i < str.length(); i++) {
			hash ^= ((hash << 5) + str.charAt(i) + (hash >> 2));
		}
		return (hash & 0x7FFFFFFF);
	}

	public static boolean isInTargetFile(String json_line_in, String file) throws IOException {
		boolean r = false;
		File f = new File(file);
		BufferedReader target_reader = new BufferedReader(new FileReader(f));

		SimpleBloomFilter bf = new SimpleBloomFilter();

		// StringBuffer list = new StringBuffer();
		String line_base;
		while ((line_base = target_reader.readLine()) != null) {
			// list.append(line_base);
			bf.add((JSON.parseObject(line_base)).getString("url"));
		}
		String url = (JSON.parseObject(json_line_in)).getString("url");

		// if (list.toString().contains(url))
		if (bf.contains(url))
			r = true;

		target_reader.close();
		return r;
	}
}
